package com.yym.cloudweb.crawl.station.dadianjing;

import org.assertj.core.util.Lists;
import org.springframework.stereotype.Service;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;

import java.util.List;

@Service
public class CrawlService {

    public static void main(String[] args) throws ClassNotFoundException, IllegalAccessException, InstantiationException {
        List<String> urlList = Lists.newArrayList();
        urlList.add("http://www.dadianjing.cn/index.php?m=Index&a=xhrList&cid=1&page=1");
        urlList.add("http://www.dadianjing.cn/index.php?m=Index&a=xhrList&cid=1&page=2");
        urlList.add("http://www.dadianjing.cn/index.php?m=Index&a=xhrList&cid=1&page=3");
        urlList.add("http://www.dadianjing.cn/index.php?m=Index&a=xhrList&cid=1&page=4");
        urlList.add("http://www.dadianjing.cn/index.php?m=Index&a=xhrList&cid=1&page=5");
        urlList.add("http://www.dadianjing.cn/index.php?m=Index&a=xhrList&cid=1&page=6");
        urlList.add("http://www.dadianjing.cn/index.php?m=Index&a=xhrList&cid=1&page=7");
        urlList.add("http://www.dadianjing.cn/index.php?m=Index&a=xhrList&cid=1&page=8");

        HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
        httpClientDownloader.setProxyProvider(SimpleProxyProvider.from(new Proxy("10.27.199.7",3128)));

        Class cla = Class.forName("com.yym.cloudweb.crawl.station.dadianjing.DadianjingPageProcessor");
        DadianjingPageProcessor dadianjingPageProcessor = (DadianjingPageProcessor)cla.newInstance();
        Spider spider = Spider.create(dadianjingPageProcessor).setDownloader(httpClientDownloader);

        for(String url : urlList){
            spider.addRequest(new Request(url));
        }

        spider.thread(4).run();
    }

}
